# -*- coding: utf-8 -*-

from scrapy import Spider, FormRequest, Request
import re
from weibosearch.items import WeiboItem
import tushare as ts

class WeiboSpider(Spider):
    name = 'weibo'
    allowed_domains = ['weibo.cn']
    start_urls = 'https://weibo.cn/search/mblog'
    max_page = 100

    def start_requests(self):
        result = ts.get_hs300s()
        keywords = result['code'].tolist()
       # keywords = ['000027', '期货']
        for keyword in keywords:
            url = '{url}?hideSearchFrame=&keyword={keyword}'.format(url=self.start_urls, keyword=keyword)
            for page in range(self.max_page + 1):
                data = {
                    'mp': str(self.max_page),
                    'page': str(page)
                }
                yield FormRequest(url, callback=self.parse_index, formdata=data, meta={'keyword': keyword})


    def parse_index(self, response):
        weibos = response.xpath('//div[@class="c" and contains(@id, "M_")]')
        for weibo in weibos:
            is_forward = bool(weibo.xpath('.//span[@class="cmt"]').extract_first())
            if is_forward:
                detail_url = weibo.xpath('.//a[contains(., "原文评论[")]//@href').extract_first()
            else:
                detail_url = weibo.xpath('.//a[contains(., "评论[")]//@href').extract_first()
            yield Request(detail_url, callback=self.parse_detail, meta={'keyword': response.meta['keyword']})



    def parse_detail(self, response):
        id = re.search('comment\/(.*?)\?', response.url).group(1)
        url = response.url
        content =''.join(response.xpath('//div[@id="M_"]//span[@class="ctt"]//text()').extract())
        print(id, url, content)
        comment_count = response.xpath('//span[@class="pms"]//text()').re_first('评论\[(.*?)\]')
        forward_count = response.xpath('//a[contains(., "转发[")]').re_first('转发\[(.*?)\]')
        like_count = response.xpath('//a[contains(., "赞[")]').re_first('赞\[(.*?)\]')
        print(comment_count, forward_count, like_count)
        posted_ad = response.xpath('//div[@id="M_"]//span[@class="ct"]//text()').extract_first(default=None)
        user = response.xpath('//div[@id="M_"]/div[1]/a/text()').extract_first(default=None)
        print(posted_ad, user)
        keyword = response.meta['keyword']
        item = WeiboItem()
        for field in item.fields:
           try:
               item[field] = eval(field)
           except NameError:
               self.logger.debug('not defing' + field)
        yield item

